"""
书生之家--浙江万里学院采集任务
http://lib.zwu.edu.cn/256/list.htm

url：http://10.60.154.22:81/book_listforward.action?kindId=0
站点资源数量：244651
采集要求：全站图书244651余本，封面跟详情页
开动时间：20180518
负责人：xujiang
"""
# 将 utils 加入到 PYTHONPATH 中
import io
import os
import re
import sys
import threading
import time
import traceback

import bs4
import chardet
from PIL import Image
from bs4 import BeautifulSoup

pwd = os.path.dirname(os.getcwd())
print(os.getcwd())
print(pwd)
utils_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".")
print(utils_path)
sys.path.insert(0, utils_path)

import utils

isbn_reg = re.compile(r'\w+-\w+-\w+-\w')
year_reg = re.compile(r"\d{4}-\d{2}-\d{2}")
publisher_reg = re.compile(r'[\u4e00-\u9fa5]+出版社')

def Img2Jpg(buf, dstFile):
    print('Img2Jpg %s ...' % dstFile)

    exMsg = ''
    try:
        srcImg = Image.open(io.BytesIO(buf))
        dstImg = srcImg.resize((108, 150), Image.ANTIALIAS).convert('RGB')
        dstImg.save(dstFile, 'JPEG')
    except:
        exMsg = '* %s' % traceback.format_exc()
        print(exMsg)
    if exMsg:
        return False

    return True

class ShuShengDownload(utils.Download):
    def __init__(self):
        self.provider = "mirrorzjwushushengbook"
        self.proxy = {"http": "http://192.168.30.176:8119"}
        self.base_url = "http://10.60.154.22:81/book_list.action?zhongtuQuery=false&kindId=&secondQuery=false&pageNo={page}"
        self.feature = '<td align="CENTER" valign="TOP">'  # 网页特征码
        super().__init__()
        super().down_list()

    def down_list(self):
        for page in range(1, 20389):
            self.down_list_run(page)

    def down_list_run(self, page):
        thread = threading.current_thread()
        filename = self.list_path + '/{page}.html'.format(page=page)
        print("{threading}:{filename}".format(threading=thread.getName(), filename=filename))
        if os.path.exists(filename):
            return
        resp = utils.get_html(self.base_url.format(page=page), proxies=self.proxy)
        if not resp:
            time.sleep(2)
            return

        if resp.content.decode("GB18030").find(self.feature) == -1:
            time.sleep(2)
            utils.printf("{threading}不包含页面特征值：{feature}".format(threading=thread.getName(), feature=self.feature))
            return
        with open(filename, mode='w', encoding='GB18030') as f:
            f.write(resp.content.decode("GB18030"))
        utils.printf("{threading}下载第{page}页完成,总共{pages}。".format(threading=thread.getName(), page=page, pages=20389))
        time.sleep(2)

    def down_detail(self, stat):
        super().down_detail()
        thread = threading.current_thread()
        conn = utils.init_db('mysql', 'zjwl')
        cur = conn.cursor()
        cur.execute("select bookid,url from shusheng where stat={}".format(stat))
        rows = cur.fetchall()
        count = len(rows)
        for bookid, baseurl in rows:
            self.down_detail_run(bookid, baseurl, conn, cur, count)
            count -= 1
        conn.close()

    def down_detail_run(self, result_queue, bookid, baseurl, conn, cur, i):
        feature = "图书详细信息"
        thread = threading.current_thread()
        path = os.path.join(self.detail_path, str(bookid)[:2])
        if not os.path.exists(path):
            os.makedirs(path)
        filename = path + "/" + str(bookid) + ".html"
        if os.path.exists(filename):
            print("{threading}:{filename}存在".format(threading=thread.getName(), filename=filename))
            sql = "update shusheng set stat=1 where bookid='{}'".format(bookid)
            print('{threading}:{sql}'.format(threading=thread.getName(), sql=sql))
            try:
                result_queue.put(sql)
                # cur.execute(sql)
                # conn.commit()
            except:
                print("{threading}bytearray index out of range".format(threading=thread.getName()))
            print("return")
            return
        url = "http://10.60.154.22:81/" + baseurl
        print("url is : {}".format(url))
        resp = utils.get_html(url, proxies=self.proxy)
        if not resp:
            time.sleep(2)
            return
        if resp.content.decode("GB18030", "ignore").find(feature) == -1:
            time.sleep(2)
            utils.printf("不包含页面特征值：{}".format(feature))
            return
        with open(filename, mode='w', encoding='GB18030') as f:
            f.write(resp.content.decode("GB18030"))
        sql = 'update shusheng set stat=1 where bookid={}'.format(bookid)
        utils.printf("{threading},{sql}".format(threading=thread.getName(), sql=sql))
        try:
            result_queue.put(sql)
            # cur.execute(sql)
            # conn.commit()
        except:
            print("{threading}bytearray index out of range".format(threading=thread.getName()))
            return
        utils.printf("{threading}下砸{bookid}成功,".format(threading=thread.getName(), bookid=bookid),
                     "任务还剩{count}".format(count=i))
        time.sleep(3)


    def down_cover(self):
        super().down_cover()
        conn = utils.init_db('mysql', 'zjwl')
        cur = conn.cursor()
        cur.execute("select bookid,cover from shusheng where cover_stat=0")
        rows = cur.fetchall()
        print(len(rows))
        for bookid, cover in rows:
            cover_path = self.cover_path + '/' + str(bookid)[:2]
            if not os.path.exists(cover_path):
                os.makedirs(cover_path)
            cover_name = cover_path + '/' + str(bookid) + '.jpg'
            if os.path.exists(cover_name):
                cur.execute("update shusheng set cover_stat=1 where bookid='{}'".format(bookid))
                conn.commit()
                continue
            resp = utils.get_html(cover, proxies=self.proxy)
            if not resp:
                cur.execute("update shusheng set cover_stat=2 where bookid='{}'".format(bookid))
                conn.commit()
                time.sleep(1)
                continue
            with open(cover_name, mode='wb') as f:
                f.write(resp.content)
            cur.execute("update shusheng set cover_stat=1 where bookid='{}'".format(bookid))
            conn.commit()
            print(cover_name)
            time.sleep(1)
        conn.close()

    # 多线程使用
    def down_cover_run(self,result_queue,bookid,cover):
        cover_path = self.cover_path + '/' + str(bookid)[:2]
        if not os.path.exists(cover_path):
            os.makedirs(cover_path)
        cover_name = cover_path + '/' + str(bookid) + '.jpg'
        print(cover_name)
        if os.path.exists(cover_name):
            print("文件存在")
            sql = "update shusheng set cover_stat=1 where bookid='{}'".format(bookid)
            result_queue.put(sql)
            return
        cover = "http://10.60.154.22:81"+cover
        print(cover)
        resp = utils.get_html(cover, proxies=self.proxy)
        if not resp:
            # 这里2代表没有封面 虽然也解析出来了封面
            sql = "update shusheng set cover_stat=2 where bookid='{}'".format(bookid)
            result_queue.put(sql)
            time.sleep(1)
            return
        if Img2Jpg(resp.content, cover_name):
            print("保存图片成功")
        else:
            print("保存文件失败,请检查")
            sql = "update shusheng set cover_stat=3 where bookid='{}'".format(bookid)
            result_queue.put(sql)
            return
        # with open(cover_name, mode='wb') as f:
        #     f.write(resp.content)
        sql = "update shusheng set cover_stat=1 where bookid='{}'".format(bookid)
        result_queue.put(sql)
        print(cover_name)
        time.sleep(1)

class ShuShengParse(utils.Parse):

    def __init__(self):
        self.provider = "mirrorzjwushushengbook"
        self.sqlLists = []
        self.count = 0
        super().__init__()

    def parse_list(self):
        super().parse_list()
        conn = utils.init_db('mysql', 'zjwl')
        cur = conn.cursor()
        url_regex = re.compile(r"(book_detail.action\?bookId=(\d+)&kindId=&kindCode=(\d+))")
        count = 0
        for file, filename in utils.file_list(self.list_path):
            # filename 包含路径
            # file 不包含路径
            with open(filename, encoding='GB18030') as f:
                txt = f.read()
            pagenumber = file.split(os.extsep)[0]
            # 通过正则获取对应的数据
            lines = url_regex.findall(txt)
            linelist = []
            for line in lines:
                line = line + (pagenumber,)
                linelist.append(line)
            lines = linelist
            # with open("./list_shusheng.txt", mode='a', encoding='utf8') as fp:
            # fp.writelines(lines)
            sql = "insert IGNORE into shusheng(url,bookid,code,page)Values(%s,%s,%s,%s)", lines
            print(sql)
            cur.executemany("insert IGNORE into shusheng(url,bookid,code,page)Values(%s,%s,%s,%s)", lines)
            count = count + 1
            utils.printf("{page}页完成,总共20388".format(page=count))
        conn.commit()
        conn.close()

    def update_cover_address(self):
        super().parse_list()
        conn = utils.init_db('mysql', 'zjwl')
        cur = conn.cursor()
        for sql in self.sqlLists:
            print(sql)
            cur.execute(sql)
        self.sqlLists = list()
        conn.commit()
        conn.close()

    def parse_cover(self):

        print(self.detail_path)
        for file, fullpath in utils.file_list(self.detail_path):
            self.count = self.count + 1
            utils.printf("{count}完成,总共244651".format(count=self.count))
            with open(fullpath, encoding='GB18030') as fp:
                txt = fp.read()
            url = self._parse_cover_one(txt)
            bookid = file.split(os.extsep)[0]
            if url:
                if url == "/images/bookdefault.gif" or url == "/images/musicdefault.gif":
                    continue
                sql = "update shusheng set cover='{}' where bookid='{}'".format(url, bookid)
                self.sqlLists.append(sql)
            if len(self.sqlLists) > 1000:
                self.update_cover_address()
        if len(self.sqlLists) > 0:
            self.update_cover_address()

    # 单线程模式
    def parse_detail(self):
        super().parse_detail()
        language = "ZH"
        type = "1"
        medium = "2"
        provider = "mirrorzjwushushengbook"
        # 国家
        country = "CN"
        # 时间 2018051700
        batch = time.strftime('%Y%m%d') + "00"
        stmt = (
            '''insert or ignore into modify_title_info_zt(lngid,rawid,title,creator,description,date,date_created,
            language,country,provider,provider_url,provider_id,type,cover,medium,batch,publisher,identifier_pisbn,title_series,folio_size)  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);'''
        )
        conn = utils.init_db('sqlite3', self.template_file)
        results = []
        cnt = 0
        # 返回文件名和全路径文件名
        for file, fullpath in utils.file_list(self.detail_path):
            # print(fullpath)
            # print(file)
            # sys.exit(-1)
        # for mun in ['99713', '99516', '99857', '99861', '99550', '99831', '99641', '99855', '99915', '99491', '99667', '9957', '99990', '99954', '9944', '9960', '99629', '99994', '99594', '99470', '99736', '99596', '99908', '99454', '99688', '99782', '99762', '99892', '99598', '99828', '99975', '9993', '99819', '99864', '99943', '9968', '99722', '99521', '99631', '99750', '99488', '9984', '99894', '99929', '99615', '99542', '99773', '99440', '99618', '99613', '99530', '9945', '9982', '99771', '99859', '99651', '99821', '99630', '99700', '99472', '9959', '99795', '99635', '99673', '9966', '99473', '99890', '9946', '99683', '99588', '99717', '99522', '99920', '99979', '99752', '99870', '9996', '99995', '99905', '99511', '99649', '99807', '99645', '99987', '99644', '99824', '99637', '99999', '99469', '99758', '99668', '99497', '99486', '99919', '99882', '99728', '99706', '99633', '99974', '99483', '99909', '99661', '99580', '99822', '99719', '99928', '99543', '99958', '99989', '99699', '99748', '99556', '99591', '99803', '99707', '99665', '99514', '99524', '99726', '99602', '99871', '99986', '99442', '99593', '99984', '99946', '99559', '9995', '99695', '99853', '99997', '99832', '99844', '99868', '99567', '99724', '99681', '99991', '99971', '99775', '99768', '99534', '99560', '9954', '99735', '99512', '99680', '99538', '99624', '99653', '99672', '99874', '99784', '99620', '99532', '99854', '99581', '99597', '99738', '99702', '9969', '99898', '99769', '99856', '99655', '99804', '99607', '99834', '99544', '99677', '9989', '99698', '99907', '9956', '99993', '99976', '99500', '99935', '99552', '99731', '99805', '99557', '99638', '99679', '99969', '99576', '99837', '99842', '99592', '99506', '9994', '99589', '99666', '99675', '9964', '99815', '99903', '99652', '99725', '99926', '99525', '99716', '99945', '99838', '99734', '9958', '9976', '99480', '99766', '99606', '99755', '99564', '99654', '99918', '99447', '99739', '99612', '99495', '99520', '99468', '99743', '99565', '99802', '99518', '99714', '99625', '99485', '9947', '9955', '99509', '99985', '9961', '99508', '99893', '99590', '99830', '99783', '99599', '99839', '99747', '99669', '99648', '99759', '99449', '9977', '99614', '99998', '99720', '99910', '99694', '99988', '99437', '99820', '99451', '999', '99587', '99650', '99577', '99462', '99646', '99881', '99921', '99476', '99973', '99962', '99636', '99682', '99659', '99547', '9979', '99705', '99949', '99888', '99459', '9967', '99601', '99634', '99729', '99545', '9985', '99448', '99711', '99899', '99732', '99493', '9980', '99622', '99896', '99457', '99481', '99553', '99977', '99917', '99944', '9965', '99492', '99466', '99878', '9953', '99471', '99678', '99551', '99887', '99749', '99740', '99456', '99603', '99690', '99499', '99767', '99458', '99568', '99426', '99513', '99877', '99948', '99757', '9943', '99867', '99623', '99691', '99964', '99501', '99484', '99571', '99996', '99527', '99533', '9963', '99549', '99889', '99927', '99886', '99640', '99561', '99902', '99519', '99660', '9962', '99689', '99505', '99697', '99983', '99562', '99494', '99464', '99733', '99895', '99911', '99753', '99965', '99883', '99658', '99439', '99818', '9949', '9950', '99498', '99961', '99756', '99611', '99939', '99438', '99829', '99937', '99566', '99980', '99751', '99535', '99574', '99900', '99563', '99737', '99897', '9983', '99461', '99555', '99836', '99796', '99951', '99845', '99966', '99647', '99585', '99833', '99676', '99776', '99710', '99674', '99901', '99788', '99841', '99627', '99664', '99657', '99741', '99754', '99696', '99558', '99639', '99578', '99693', '99884', '99569', '99972', '99742', '99978']:
        #     fullpath = "D:\\project\\python\\download\\mirrorzjwushushengbook\\detail\\update"
        #     fullpath = fullpath+ "\\"+mun[:2]+"\\"+mun+".html"
        #     file = mun+".html"
            with open(fullpath, encoding='GB18030') as fp:
                txt = fp.read()
            (title, description, date_created, publisher, isbn, title_series, creator, folio_size) = self._parse_detail_one(txt)
            print(self._parse_detail_one(txt))
            # 分割文件，取前面部分
            basename, _, _ = file.partition('.')
            rawid = basename
            # 取年份
            if date_created:
                date = date_created[:4]
            else:
                date = ''
            lngid = "MIRROR_ZJWU_SHUSHENG_TS_" + rawid
            provider_url = provider + "@http://10.60.154.22:81/book_detail.action?bookId=" + rawid
            provider_id = provider + "@" + rawid
            #  不太
            cover_file = rawid[:2] + "/" + rawid + '.jpg'
            if os.path.exists(os.path.join(self.cover_path, cover_file)):
                cover = '/smartlib/' + provider + '/' + cover_file
            else:
                cover = ''
            results.append((
                lngid, rawid, title, creator, description, date, date_created, language, country, provider,
                provider_url, provider_id, type, cover, medium, batch, publisher, isbn, title_series, folio_size))
            if utils.parse_results_to_sql(conn, stmt, results, 1000):
                cnt += 1001
                utils.printf("已经解析大约 ", cnt, ' 条数据...')
                results.clear()

        utils.parse_results_to_sql(conn, stmt, results)
        cnt += len(results)
        utils.printf("已经解析 ", cnt, ' 条数据...')
        conn.close()

    # 多线程模式
    def parse_detail_run(self, result_queue, file, fullpath):
        super().parse_detail()
        language = "ZH"
        type = "1"
        medium = "2"
        provider = "mirrorzjwushushengbook"
        # 国家
        country = "CN"
        # 时间 2018051700
        batch = time.strftime('%Y%m%d') + "00"
        # stmt = (
        #     '''insert or ignore into modify_title_info_zt(lngid,rawid,title,creator,description,date,date_created,
        #     language,country,provider,provider_url,provider_id,type,cover,medium,batch,publisher,identifier_pisbn,title_series)  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);'''
        # )
        # conn = utils.init_db('sqlite3', self.template_file)
        # results = []
        cnt = 0
        # # 返回文件名和全路径文件名
        # for file, fullpath in utils.file_list(self.detail_path):
        with open(fullpath, encoding='GB18030') as fp:
            txt = fp.read()
        (title, description, date_created, publisher, isbn, title_series, creator, folio_size) = self._parse_detail_one(txt)
        print(self._parse_detail_one(txt))
        # 分割文件，取前面部分
        basename, _, _ = file.partition('.')
        rawid = basename
        # 取年份
        if date_created:
            date = date_created[:4]
        else:
            date = ''
        lngid = "MIRROR_ZJWU_SHUSHENG_TS_" + rawid
        provider_url = provider + "@http://10.60.154.22:81/book_detail.action?bookId=" + rawid
        provider_id = provider + "@" + rawid
        #  不太
        cover_file = rawid[:2] + "/" + rawid + '.jpg'
        if os.path.exists(os.path.join(self.cover_path, cover_file)):
            cover = '/smartlib/' + provider + '/' + cover_file
        else:
            cover = ''
        result_queue.put((
            lngid, rawid, title, creator, description, date, date_created, language, country, provider,
            provider_url, provider_id, type, cover, medium, batch, publisher, isbn, title_series, folio_size))
        # if utils.parse_results_to_sql(conn, stmt, results, 1000):
        #     cnt += 1001
        #     utils.printf("已经解析大约 ", cnt, ' 条数据...')
        #     results.clear()

        # utils.parse_results_to_sql(conn, stmt, results)
        # cnt += len(results)
        # utils.printf("已经解析 ", cnt, ' 条数据...')
        # conn.close()

    def _parse_cover_one(self, txt):
        soup = BeautifulSoup(txt, 'lxml')
        tr_tag = soup.find("tr", valign="top")
        if tr_tag:
            imag_tag = tr_tag.find("img")
            if imag_tag:
                url = imag_tag['src']
                print(url)
                return url
            else:
                print("没有发现这个tr标签")
                sys.exit(-1)
        else:
            print("没有发现这个tr标签")
            sys.exit(-1)
        return False


    def _parse_detail_one(self, txt):
        # 书名
        title = ""
        #摘要
        description = ""
        date_created = ""
        publisher = ""
        isbn = ""
        # 丛书
        title_series = ""
        # 作者
        creator = ""
        # 开本
        folio_size = ""
        soup = BeautifulSoup(txt, 'lxml')
        intro_tag = soup.select_one('table[height="130"] td[class="12Black"]')
        # 包括书名 作者 开本 在内的很多信息块
        title_creator_tag = soup.select_one('tr[valign="MIDDLE"] > td[class="12Black"]')
        try:
            # 内容摘要
            description = intro_tag.string.strip()
            if re.match("^\d\\\d*", description).group():
                description = ''
        except AttributeError as e:
            pass
        # 目前本页strong标签 书名 作者 开本
        for xx in title_creator_tag.find_all("strong"):
            # print(title_creator_tag.find_all("strong"))
            if xx.string.strip().find("丛书") >= 0:
                title_series = xx.next_sibling.strip()
            elif xx.string.strip().find("书名") >= 0:
                title = xx.next_sibling.strip()
                title = title.replace("《", "").replace("》", "")
            elif xx.string.strip().find("作者") >= 0:
                creator = xx.next_sibling.strip()
                creator = creator.replace(",", ";")
            elif xx.string.strip().find("开本") >= 0:
                folio_size = xx.next_sibling.strip()
        for item in title_creator_tag.contents:
            # bs4.element.Comment为注释类型
            if not isinstance(item, bs4.element.Comment):
                continue
            # 匹配注释中的ISBN号
            isbn_tmp = isbn_reg.findall(item.string)
            # 匹配注释中的日期及出版日期
            year_tmp = year_reg.findall(item.string)
            # 匹配注释中的出版社
            publisher_tmp = publisher_reg.findall(item.string)
            if isbn_tmp:
                isbn = isbn_tmp[0]
            if year_tmp:
                date_created = year_tmp[0].replace("-", "")
            if publisher_tmp:
                publisher = publisher_tmp[0]

        return title, description, date_created, publisher, isbn.replace('-', ''), title_series, creator,folio_size


if __name__ == "__main__":
    down = ShuShengDownload()
    # down.down_list()
    parse = ShuShengParse()
    # parse.parse_list()
    # down.down_detail(0)
    parse.parse_detail()
    # parse.parse_cover()
    # down.down_cover()
